(** 
    Page.ml

    Bozman Çagdas <bozman.cagdas@gmail.com> - 2011
    Razakamiadana Tefin <tefy.razaka@gmail.com>   - 2011
*)

open Utils
open Sets

type page = string * string* string

module PageSet = Set.Make (
  struct 
    type t = page
    let compare = Pervasives.compare
  end)

let id = "pageid"
let title = "title"
let qcont = "query-continue"
let blcont = "blcontinue"
let gplcont = "gplcontinue"
let lang = "fr"
let http = spf "http://%s.wikipedia.org/w/api.php?" lang
let action = "query"
let parse = "parse"
let wikitext = "wikitext"
let format = "xml"
let generator = "links"
let prop = "info"
let list = "backlinks"
let limit = 650

let request = HttpRequest.string_of_url

let init title : page =
  let title = format_title title in
  let backlink = spf "%saction=%s&prop=%s&list=%s&format=%s&bltitle=%s&bllimit=%d" 
    http action prop list format title limit in  
  let link = spf "%saction=%s&format=%s&generator=%s&titles=%s&prop=%s&gpllimit=%d"
    http action format generator title prop limit in
  let text = spf "%saction=%s&format=%s&prop=%s&page=%s"
    http parse format wikitext title in
  debug "     link : %s\n backlink : %s\nwiki-text : %s\n"
    link backlink text; flush stderr;
  link, backlink, text

let blcontinue backlink cont =
  let f_cont = format_title cont in
  let new_link = spf "%s&blcontinue=%s" backlink f_cont in
  debug "     ... blcontinue : %s\r" f_cont; flush stderr;
  let s = request new_link in
  XMLParser.attribs_list s 

let gplcontinue link cont =
  let f_cont = format_title cont in
  let new_link = spf "%s&gplcontinue=%s" link f_cont in
  debug "     ... gplcontinue : %s\r" f_cont; flush stderr;
  let s = request new_link in
  XMLParser.attribs_list s 

let backlinks_ids page =
  let link = (fun (_, bl, _) -> bl) page in  
  let request_result = request link in
  let backward = XMLParser.attribs_list request_result in
  let rec aux page =
    let set, add, union = IntSet.empty, IntSet.add, IntSet.union in
    List.fold_left
      (fun set l ->
	List.fold_left 
	  (fun set (attr, value) ->
	    if attr = blcont then
	      let new_page = blcontinue link value in
	      union set (aux new_page)
	    else if attr = id then add (int_of_string value) set
	    else set)
	  set
	  l)
      set
      page
  in
  aux backward

let backlinks_titles page = 
  let link = (fun (_, bl, _) -> bl) page in
  let request_result = request link in
  let backward = XMLParser.attribs_list request_result in
  let rec aux page =
    let set, add, union = StringSet.empty, StringSet.add, StringSet.union in
    List.fold_left
      (fun set l ->
	List.fold_left 
	  (fun set (attr, value) ->
	    if attr = blcont then
	      let new_page = blcontinue link value in
	      union set (aux new_page)
	    else if attr = title then add value set
	    else set)
	  set
	  l)
      set
      page
  in
  aux backward
    
let inlinks_ids page =
  let link = (fun (il, _, _) -> il) page in
  let request_result = request link in
  let forward = XMLParser.attribs_list request_result in
  let rec aux page =
    let set, add, union = IntSet.empty, IntSet.add, IntSet.union in
    List.fold_left
      (fun set l ->
	List.fold_left 
	  (fun set (attr, value) ->
	    if attr = gplcont then 	      
	      let new_page = gplcontinue link value in
	      union set (aux new_page)
	    else if attr = id then add (int_of_string value) set
	    else set)
	  set
	  l)
      set
      page
  in
  aux forward

let inlinks_titles page =
  let link = (fun (il, _, _) -> il) page in
  let request_result = request link in
  let forward = XMLParser.attribs_list request_result in
  let rec aux page =
    let set, add, union = StringSet.empty, StringSet.add, StringSet.union in
    List.fold_left
      (fun set l ->
	List.fold_left 
	  (fun set (attr, value) ->
	    if attr = gplcont then 
	      let new_page = gplcontinue link value in 
	      union set (aux new_page)
	    else if attr = title then add value set
	    else set)
	  set
	  l)
      set
      page
  in
  aux forward  

let wiki_text page =
  let link =  (fun (_, _, wl) -> wl) page in
  let request_result = request link in
  XMLParser.wiki_text request_result
